Home Erkunden Hilfe
Registrieren Anmelden
J1X
/
huginn
Beobachten 2
Markieren 0
Fork 0

Quellcode durchsuchen
Rename max_assignments to assignments; add lots of validations and tests

Andrew Cantino vor 12 Jahren
Ursprung
e24f940ac8
Commit
31cdf610ac
2 geänderte Dateien mit 112 neuen Zeilen und 6 gelöschten Zeilen
Diff-Statistik anzeigen
  1. 19 4
      app/models/agents/human_task_agent.rb
  2. 93 2
      spec/models/agents/human_task_agent_spec.rb

+ 19 - 4
app/models/agents/human_task_agent.rb
Datei anzeigen 

@@ -19,7 +19,7 @@ module Agents
19 19 
             "expected_receive_period_in_days": 2,
20 20 
             "trigger_on": "event",
21 21 
             "hit": {
22 
-              "max_assignments": 1,
22 
+              "assignments": 1,
23 23 
               "title": "Sentiment evaluation",
24 24 
               "description": "Please rate the sentiment of this message: '<$.message>'",
25 25 
               "reward": 0.05,
 
@@ -58,7 +58,7 @@ module Agents
58 58 
       `default`, `min_length`, and `max_length`.
59 59
60 60 
       If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to
61 
-      automatically select the majority vote for each question across all `max_assignments`.
61 
+      automatically select the majority vote for each question across all `assignments`.  If all selections are numeric, an `average_answer` will also be generated.
62 62
63 63 
       As with most Agents, `expected_receive_period_in_days` is required if `trigger_on` is set to `event`.
64 64 
     MD
 
@@ -71,7 +71,14 @@ module Agents
71 71 
     MD
72 72
73 73 
     def validate_options
74 
+      options[:hit] ||= {}
75 
+      options[:hit][:questions] ||= []
76 
+
74 77 
       errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on])
78 
+      errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options[:hit][:assignments].present? && options[:hit][:assignments].to_i > 0
79 
+      errors.add(:base, "'hit.title' must be provided") unless options[:hit][:title].present?
80 
+      errors.add(:base, "'hit.description' must be provided") unless options[:hit][:description].present?
81 
+      errors.add(:base, "'hit.questions' must be provided") unless options[:hit][:questions].present? && options[:hit][:questions].length > 0
75 82
76 83 
       if options[:trigger_on] == "event"
77 84 
         errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present?
 
@@ -79,6 +86,14 @@ module Agents
79 86 
         errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0
80 87 
       end
81 88
89 
+      if options[:hit][:questions].any? { |question| [:key, :name, :required, :type, :question].any? {|k| !question[k].present? } }
90 
+        errors.add(:base, "all questions must set 'key', 'name', 'required', 'type', and 'question'")
91 
+      end
92 
+
93 
+      if options[:hit][:questions].any? { |question| question[:type] == "selection" && (!question[:selections].present? || question[:selections].length == 0 || !question[:selections].all? {|s| s[:key].present? } || !question[:selections].all? { |s| s[:text].present? })}
94 
+        errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'")
95 
+      end
96 
+
82 97 
       if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" }
83 98 
         errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
84 99 
       end
 
@@ -90,7 +105,7 @@ module Agents
90 105 
         :trigger_on => "event",
91 106 
         :hit =>
92 107 
           {
93 
-            :max_assignments => 1,
108 
+            :assignments => 1,
94 109 
             :title => "Sentiment evaluation",
95 110 
             :description => "Please rate the sentiment of this message: '<$.message>'",
96 111 
             :reward => 0.05,
 
@@ -225,7 +240,7 @@ module Agents
225 240 
       description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip
226 241 
       questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload)
227 242 
       hit = RTurk::Hit.create(:title => title) do |hit|
228 
-        hit.max_assignments = (options[:hit][:max_assignments] || 1).to_i
243 
+        hit.max_assignments = (options[:hit][:assignments] || 1).to_i
229 244 
         hit.description = description
230 245 
         hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
231 246 
         hit.reward = (options[:hit][:reward] || 0.05).to_f

+ 93 - 2
spec/models/agents/human_task_agent_spec.rb
Datei anzeigen 

@@ -17,6 +17,97 @@ describe Agents::HumanTaskAgent do
17 17 
   end
18 18
19 19 
   describe "validations" do
20 
+    it "validates that trigger_on is 'schedule' or 'event'" do
21 
+      @checker.options[:trigger_on] = "foo"
22 
+      @checker.should_not be_valid
23 
+    end
24 
+
25 
+    it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do
26 
+      @checker.options[:trigger_on] = "event"
27 
+      @checker.options[:expected_receive_period_in_days] = nil
28 
+      @checker.should_not be_valid
29 
+      @checker.options[:expected_receive_period_in_days] = 2
30 
+      @checker.should be_valid
31 
+    end
32 
+
33 
+    it "requires a positive submission_period when trigger_on is set to 'schedule'" do
34 
+      @checker.options[:trigger_on] = "schedule"
35 
+      @checker.options[:submission_period] = nil
36 
+      @checker.should_not be_valid
37 
+      @checker.options[:submission_period] = 2
38 
+      @checker.should be_valid
39 
+    end
40 
+
41 
+    it "requires a hit.title" do
42 
+      @checker.options[:hit][:title] = ""
43 
+      @checker.should_not be_valid
44 
+    end
45 
+
46 
+    it "requires a hit.description" do
47 
+      @checker.options[:hit][:description] = ""
48 
+      @checker.should_not be_valid
49 
+    end
50 
+
51 
+    it "requires hit.assignments" do
52 
+      @checker.options[:hit][:assignments] = ""
53 
+      @checker.should_not be_valid
54 
+      @checker.options[:hit][:assignments] = 0
55 
+      @checker.should_not be_valid
56 
+      @checker.options[:hit][:assignments] = "moose"
57 
+      @checker.should_not be_valid
58 
+      @checker.options[:hit][:assignments] = "2"
59 
+      @checker.should be_valid
60 
+    end
61 
+
62 
+    it "requires hit.questions" do
63 
+      old_questions = @checker.options[:hit][:questions]
64 
+      @checker.options[:hit][:questions] = nil
65 
+      @checker.should_not be_valid
66 
+      @checker.options[:hit][:questions] = []
67 
+      @checker.should_not be_valid
68 
+      @checker.options[:hit][:questions] = [old_questions[0]]
69 
+      @checker.should be_valid
70 
+    end
71 
+
72 
+    it "requires that all questions have key, name, required, type, and question" do
73 
+      old_questions = @checker.options[:hit][:questions]
74 
+      @checker.options[:hit][:questions].first[:key] = ""
75 
+      @checker.should_not be_valid
76 
+
77 
+      @checker.options[:hit][:questions] = old_questions
78 
+      @checker.options[:hit][:questions].first[:name] = ""
79 
+      @checker.should_not be_valid
80 
+
81 
+      @checker.options[:hit][:questions] = old_questions
82 
+      @checker.options[:hit][:questions].first[:required] = nil
83 
+      @checker.should_not be_valid
84 
+
85 
+      @checker.options[:hit][:questions] = old_questions
86 
+      @checker.options[:hit][:questions].first[:type] = ""
87 
+      @checker.should_not be_valid
88 
+
89 
+      @checker.options[:hit][:questions] = old_questions
90 
+      @checker.options[:hit][:questions].first[:question] = ""
91 
+      @checker.should_not be_valid
92 
+    end
93 
+
94 
+    it "requires that all questions of type 'selection' have a selections array with keys and text" do
95 
+      @checker.options[:hit][:questions][0][:selections] = []
96 
+      @checker.should_not be_valid
97 
+      @checker.options[:hit][:questions][0][:selections] = [{}]
98 
+      @checker.should_not be_valid
99 
+      @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "" }]
100 
+      @checker.should_not be_valid
101 
+      @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "hi" }]
102 
+      @checker.should_not be_valid
103 
+      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "" }]
104 
+      @checker.should_not be_valid
105 
+      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }]
106 
+      @checker.should be_valid
107 
+      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }, {}]
108 
+      @checker.should_not be_valid
109 
+    end
110 
+
20 111 
     it "requires that all questions be of type 'selection' when `take_majority` is `true`" do
21 112 
       @checker.options[:take_majority] = "true"
22 113 
       @checker.should_not be_valid
 
@@ -91,7 +182,7 @@ describe Agents::HumanTaskAgent do
91 182
92 183 
       @checker.send :create_hit, @event
93 184
94 
-      hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments]
185 
+      hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
95 186 
       hitInterface.reward.should == @checker.options[:hit][:reward]
96 187 
       hitInterface.description.should == "Make something for Joe"
97 188 
 
@@ -110,7 +201,7 @@ describe Agents::HumanTaskAgent do
110 201 
       mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
111 202 
       mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
112 203 
       @checker.send :create_hit
113 
-      hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments]
204 
+      hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
114 205 
       hitInterface.reward.should == @checker.options[:hit][:reward]
115 206 
     end
116 207 
   end